{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d87c59c0-bd47-4b9e-9b80-96b59e4f0d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "from feature_utils import FNAMES,COLUMNS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "68b657c8-c16e-4d8c-9843-0a3543247852",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset(data_dir = 'out'):\n",
    "    dfiles = os.listdir(data_dir)\n",
    "    absfiles = [os.path.join(data_dir,f) for f in dfiles]\n",
    "    dfs = []\n",
    "    for af in absfiles:\n",
    "        df = pd.read_csv(af, names = COLUMNS, header = None)\n",
    "        dfs.append(df)\n",
    "    df = pd.concat(dfs)\n",
    "    df['dt'] = pd.to_datetime(df.date)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8e18831b-fe04-4f4a-9eb9-64bf540f1732",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = get_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f4fcafb8-9989-4f11-b10d-a8fd7f3de456",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(           date         code name     rsi_6    rsi_12    rsi_24   stoch_k  \\\n",
       " 334  2021-07-23  002626.XSHE  金达威  0.131833  0.255450  0.371504  0.042292   \n",
       " 335  2021-07-26  002626.XSHE  金达威  0.100604  0.227204  0.352797  0.045783   \n",
       " 336  2021-08-16  002626.XSHE  金达威  0.539104  0.422497  0.406209  0.441895   \n",
       " 337  2021-08-17  002626.XSHE  金达威  0.365084  0.351049  0.374906  0.422782   \n",
       " 338  2021-08-25  002626.XSHE  金达威  0.615762  0.501203  0.443837  0.629013   \n",
       " \n",
       "       stoch_d   stoch_j       bbp    ...      SMA_DIFF_13  VOL_SMA_DIFF_5  \\\n",
       " 334  0.082808 -0.038741  0.097891    ...        -0.068220        0.370872   \n",
       " 335  0.066565  0.004218  0.048450    ...        -0.085332       -0.047792   \n",
       " 336  0.427633  0.470418  0.463674    ...         0.013109        0.135324   \n",
       " 337  0.409100  0.450146  0.209175    ...        -0.023016        0.091329   \n",
       " 338  0.412798  1.061444  0.721654    ...         0.026659        0.188244   \n",
       " \n",
       "      VOL_SMA_DIFF_13      BETA    CORREL    STDDEV       CCI     WILLR  \\\n",
       " 334         0.536186  0.357855  0.984032  0.742956 -1.766051 -0.993377   \n",
       " 335         0.086560  0.559068  0.986147  1.124377 -1.892520 -0.946844   \n",
       " 336         0.155444  0.776754  0.982970  0.469425  0.572757 -0.251701   \n",
       " 337         0.051776  0.889989  0.989156  0.482643 -0.911689 -0.866109   \n",
       " 338         0.247297  1.051644  0.981017  1.122807  0.983129 -0.192857   \n",
       " \n",
       "      target         dt  \n",
       " 334     0.0 2021-07-23  \n",
       " 335     0.0 2021-07-26  \n",
       " 336     0.0 2021-08-16  \n",
       " 337     0.0 2021-08-17  \n",
       " 338     0.0 2021-08-25  \n",
       " \n",
       " [5 rows x 25 columns],\n",
       " (234125, 25))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.tail(), ds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "68dc7eab-f5b6-4d6e-8b42-316afded095e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rsi_6</th>\n",
       "      <th>rsi_12</th>\n",
       "      <th>rsi_24</th>\n",
       "      <th>stoch_k</th>\n",
       "      <th>stoch_d</th>\n",
       "      <th>stoch_j</th>\n",
       "      <th>bbp</th>\n",
       "      <th>b_up_low</th>\n",
       "      <th>macd</th>\n",
       "      <th>macdsignal</th>\n",
       "      <th>...</th>\n",
       "      <th>SMA_DIFF_5</th>\n",
       "      <th>SMA_DIFF_13</th>\n",
       "      <th>VOL_SMA_DIFF_5</th>\n",
       "      <th>VOL_SMA_DIFF_13</th>\n",
       "      <th>BETA</th>\n",
       "      <th>CORREL</th>\n",
       "      <th>STDDEV</th>\n",
       "      <th>CCI</th>\n",
       "      <th>WILLR</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>2.341250e+05</td>\n",
       "      <td>2.341250e+05</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "      <td>234125.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.469617</td>\n",
       "      <td>0.489421</td>\n",
       "      <td>0.500438</td>\n",
       "      <td>4.670684e-01</td>\n",
       "      <td>4.843578e-01</td>\n",
       "      <td>0.432490</td>\n",
       "      <td>0.459011</td>\n",
       "      <td>1.221463</td>\n",
       "      <td>0.106282</td>\n",
       "      <td>0.129517</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.007779</td>\n",
       "      <td>-0.006522</td>\n",
       "      <td>0.046577</td>\n",
       "      <td>0.083563</td>\n",
       "      <td>0.595578</td>\n",
       "      <td>0.928413</td>\n",
       "      <td>0.491463</td>\n",
       "      <td>-0.127854</td>\n",
       "      <td>-0.563688</td>\n",
       "      <td>0.127645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.184678</td>\n",
       "      <td>0.134256</td>\n",
       "      <td>0.098538</td>\n",
       "      <td>2.599083e-01</td>\n",
       "      <td>2.493654e-01</td>\n",
       "      <td>0.358831</td>\n",
       "      <td>0.340712</td>\n",
       "      <td>3.017041</td>\n",
       "      <td>1.834502</td>\n",
       "      <td>1.679282</td>\n",
       "      <td>...</td>\n",
       "      <td>0.036211</td>\n",
       "      <td>0.065652</td>\n",
       "      <td>0.457314</td>\n",
       "      <td>0.804487</td>\n",
       "      <td>0.796321</td>\n",
       "      <td>0.075325</td>\n",
       "      <td>1.555918</td>\n",
       "      <td>1.140252</td>\n",
       "      <td>0.285644</td>\n",
       "      <td>0.333695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3.505344e-15</td>\n",
       "      <td>-4.981694e-15</td>\n",
       "      <td>-0.666667</td>\n",
       "      <td>-0.589725</td>\n",
       "      <td>-1440.456616</td>\n",
       "      <td>-105.391668</td>\n",
       "      <td>-80.477146</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.318508</td>\n",
       "      <td>-0.470331</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-61.036817</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-4.666667</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.333489</td>\n",
       "      <td>0.397013</td>\n",
       "      <td>0.434678</td>\n",
       "      <td>2.307692e-01</td>\n",
       "      <td>2.616677e-01</td>\n",
       "      <td>0.129163</td>\n",
       "      <td>0.189513</td>\n",
       "      <td>1.114776</td>\n",
       "      <td>-0.149051</td>\n",
       "      <td>-0.129260</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.025235</td>\n",
       "      <td>-0.038326</td>\n",
       "      <td>-0.221749</td>\n",
       "      <td>-0.285261</td>\n",
       "      <td>0.263390</td>\n",
       "      <td>0.908639</td>\n",
       "      <td>0.084475</td>\n",
       "      <td>-0.993640</td>\n",
       "      <td>-0.823529</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.458348</td>\n",
       "      <td>0.483758</td>\n",
       "      <td>0.495754</td>\n",
       "      <td>4.614401e-01</td>\n",
       "      <td>4.890313e-01</td>\n",
       "      <td>0.409722</td>\n",
       "      <td>0.439970</td>\n",
       "      <td>1.172760</td>\n",
       "      <td>0.010209</td>\n",
       "      <td>0.017587</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.007154</td>\n",
       "      <td>-0.007471</td>\n",
       "      <td>-0.038787</td>\n",
       "      <td>-0.072515</td>\n",
       "      <td>0.573355</td>\n",
       "      <td>0.947362</td>\n",
       "      <td>0.188319</td>\n",
       "      <td>-0.182930</td>\n",
       "      <td>-0.582996</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.599761</td>\n",
       "      <td>0.578009</td>\n",
       "      <td>0.562033</td>\n",
       "      <td>6.996241e-01</td>\n",
       "      <td>7.066515e-01</td>\n",
       "      <td>0.733564</td>\n",
       "      <td>0.731051</td>\n",
       "      <td>1.269547</td>\n",
       "      <td>0.207172</td>\n",
       "      <td>0.211443</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010827</td>\n",
       "      <td>0.024740</td>\n",
       "      <td>0.205235</td>\n",
       "      <td>0.234578</td>\n",
       "      <td>0.897126</td>\n",
       "      <td>0.970886</td>\n",
       "      <td>0.435321</td>\n",
       "      <td>0.755285</td>\n",
       "      <td>-0.311224</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.602740</td>\n",
       "      <td>1.589725</td>\n",
       "      <td>87.985701</td>\n",
       "      <td>125.435349</td>\n",
       "      <td>99.889790</td>\n",
       "      <td>...</td>\n",
       "      <td>2.697917</td>\n",
       "      <td>5.321918</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>99.264963</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>125.943485</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>-0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               rsi_6         rsi_12         rsi_24       stoch_k  \\\n",
       "count  234125.000000  234125.000000  234125.000000  2.341250e+05   \n",
       "mean        0.469617       0.489421       0.500438  4.670684e-01   \n",
       "std         0.184678       0.134256       0.098538  2.599083e-01   \n",
       "min         0.000000       0.000000       0.000000 -3.505344e-15   \n",
       "25%         0.333489       0.397013       0.434678  2.307692e-01   \n",
       "50%         0.458348       0.483758       0.495754  4.614401e-01   \n",
       "75%         0.599761       0.578009       0.562033  6.996241e-01   \n",
       "max         1.000000       1.000000       1.000000  1.000000e+00   \n",
       "\n",
       "            stoch_d        stoch_j            bbp       b_up_low  \\\n",
       "count  2.341250e+05  234125.000000  234125.000000  234125.000000   \n",
       "mean   4.843578e-01       0.432490       0.459011       1.221463   \n",
       "std    2.493654e-01       0.358831       0.340712       3.017041   \n",
       "min   -4.981694e-15      -0.666667      -0.589725   -1440.456616   \n",
       "25%    2.616677e-01       0.129163       0.189513       1.114776   \n",
       "50%    4.890313e-01       0.409722       0.439970       1.172760   \n",
       "75%    7.066515e-01       0.733564       0.731051       1.269547   \n",
       "max    1.000000e+00       1.602740       1.589725      87.985701   \n",
       "\n",
       "                macd     macdsignal      ...           SMA_DIFF_5  \\\n",
       "count  234125.000000  234125.000000      ...        234125.000000   \n",
       "mean        0.106282       0.129517      ...            -0.007779   \n",
       "std         1.834502       1.679282      ...             0.036211   \n",
       "min      -105.391668     -80.477146      ...            -0.318508   \n",
       "25%        -0.149051      -0.129260      ...            -0.025235   \n",
       "50%         0.010209       0.017587      ...            -0.007154   \n",
       "75%         0.207172       0.211443      ...             0.010827   \n",
       "max       125.435349      99.889790      ...             2.697917   \n",
       "\n",
       "         SMA_DIFF_13  VOL_SMA_DIFF_5  VOL_SMA_DIFF_13           BETA  \\\n",
       "count  234125.000000   234125.000000    234125.000000  234125.000000   \n",
       "mean       -0.006522        0.046577         0.083563       0.595578   \n",
       "std         0.065652        0.457314         0.804487       0.796321   \n",
       "min        -0.470331       -1.000000        -1.000000     -61.036817   \n",
       "25%        -0.038326       -0.221749        -0.285261       0.263390   \n",
       "50%        -0.007471       -0.038787        -0.072515       0.573355   \n",
       "75%         0.024740        0.205235         0.234578       0.897126   \n",
       "max         5.321918        4.000000        12.000000      99.264963   \n",
       "\n",
       "              CORREL         STDDEV            CCI          WILLR  \\\n",
       "count  234125.000000  234125.000000  234125.000000  234125.000000   \n",
       "mean        0.928413       0.491463      -0.127854      -0.563688   \n",
       "std         0.075325       1.555918       1.140252       0.285644   \n",
       "min        -1.000000       0.000000      -4.666667      -1.000000   \n",
       "25%         0.908639       0.084475      -0.993640      -0.823529   \n",
       "50%         0.947362       0.188319      -0.182930      -0.582996   \n",
       "75%         0.970886       0.435321       0.755285      -0.311224   \n",
       "max         1.000000     125.943485       4.666667      -0.000000   \n",
       "\n",
       "              target  \n",
       "count  234125.000000  \n",
       "mean        0.127645  \n",
       "std         0.333695  \n",
       "min         0.000000  \n",
       "25%         0.000000  \n",
       "50%         0.000000  \n",
       "75%         0.000000  \n",
       "max         1.000000  \n",
       "\n",
       "[8 rows x 21 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "efbad786-ddad-4e3b-be20-519840842e04",
   "metadata": {},
   "outputs": [],
   "source": [
    "p = ds[ds.target == 1]\n",
    "n = ds[ds.target == 0]\n",
    "pos,neg = p.shape[0],n.shape[0]\n",
    "total = pos +neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "49858c42-951a-4f3a-9972-7f31e622c956",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.12764548852108917, 0.8723545114789109)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos/total, neg/ total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c145570a-0e67-426b-a436-3123aaecf493",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((173876, 25), (60249, 25))"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds = ds[ds.dt < '2019-01-01']\n",
    "test_ds  = ds[ds.dt >= '2019-01-01']\n",
    "train_ds.shape, test_ds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "dde0ca1a-4ea8-435a-ba42-3618a682e8ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "df43c840-98d8-47e7-a272-37d4a600d313",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:765: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "lr.fit(train_ds[FNAMES], train_ds['target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "71c060c6-d343-4cc4-abdd-6d4a5729325a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.93092538])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "4e1aee2e-a741-497a-93d8-0ac929c48438",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'rsi_6': -1.0639237061502664,\n",
       " 'rsi_12': -2.795057755139309,\n",
       " 'rsi_24': 0.5905798609347783,\n",
       " 'stoch_k': 0.08545482470928704,\n",
       " 'stoch_d': 1.2781808246727628,\n",
       " 'stoch_j': -2.2999971752176727,\n",
       " 'bbp': 4.122474219176075,\n",
       " 'b_up_low': 0.10470882052228203,\n",
       " 'macd': -0.44462502395455344,\n",
       " 'macdsignal': -0.11186150003493452,\n",
       " 'macdhist': -0.33276352391959474,\n",
       " 'SMA_DIFF_5': 0.967690704107216,\n",
       " 'SMA_DIFF_13': -5.751094661025108,\n",
       " 'VOL_SMA_DIFF_5': 0.27933333419365297,\n",
       " 'VOL_SMA_DIFF_13': -0.28012966459405453,\n",
       " 'BETA': -0.09234207463004125,\n",
       " 'CORREL': 1.066559992619457,\n",
       " 'STDDEV': -0.6671747324253127,\n",
       " 'CCI': -1.631762110125724,\n",
       " 'WILLR': 2.9274549690497094}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict(zip(FNAMES, lr.coef_[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8990ab6-09c2-4eda-bca0-aa9f073660b4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed57cd91-b1fd-4c62-8dd7-5996d3fb2bf6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
